package tetris.rlpark_wrapper;

import java.util.Random;

import rlpark.plugin.rltoys.algorithms.control.acting.EpsilonGreedy;
import rlpark.plugin.rltoys.algorithms.control.sarsa.Sarsa;
import rlpark.plugin.rltoys.algorithms.control.sarsa.SarsaControl;
import rlpark.plugin.rltoys.algorithms.functions.stateactions.TabularAction;
import rlpark.plugin.rltoys.algorithms.traces.RTraces;
import rlpark.plugin.rltoys.envio.actions.Action;
import rlpark.plugin.rltoys.envio.policy.Policy;
import rlpark.plugin.rltoys.envio.rl.TRStep;
import rlpark.plugin.rltoys.math.vector.RealVector;
import rlpark.plugin.rltoys.math.vector.implementations.Vectors;

import tetris.rlpark_wrapper.TetrisProblem;
import rlpark.plugin.rltoys.algorithms.representations.tilescoding.TileCodersNoHashing;
import rlpark.plugin.rltoys.algorithms.representations.rbf.RBFs;

public class SarsaTetris implements Runnable {
  private final TetrisProblem problem;
  private final SarsaControl control;
  private final TileCodersNoHashing projector;
  //private final RBFs projector;
  
  public SarsaTetris() {
    problem = new TetrisProblem();
    
    projector = new TileCodersNoHashing(problem.getObservationRanges());
    projector.addFullTilings(3, 10);
    projector.includeActiveFeature();

    /*
    projector = new RBFs(1.0);
    projector.addFullRBFs(problem.getObservationRanges(), 3,  1);
    projector.includeActiveFeature();*/
    
    TabularAction toStateAction = new TabularAction(problem.actions(), projector.vectorNorm(), projector.vectorSize());
    toStateAction.includeActiveFeature();
    double alpha = .5 / projector.vectorNorm();
    double gamma = 0.99;
    double lambda = 0.3;
    Sarsa sarsa = new Sarsa(alpha, gamma, lambda, toStateAction.vectorSize(), new RTraces());
    double epsilon = 0.01;
    Policy acting = new EpsilonGreedy(new Random(0), problem.actions(), toStateAction, sarsa, epsilon);
    control = new SarsaControl(acting, toStateAction, sarsa);
  }

  @Override
  public void run() {
    TRStep step = problem.initialize();
    int nbEpisode = 0;
    RealVector x_t = null;
    while (true) {
      RealVector x_tp1 = projector.project(step.o_tp1);
      Action action = control.step(x_t, step.a_t, x_tp1, step.r_tp1);
      x_t = Vectors.bufferedCopy(x_tp1, x_t);
      if (step.isEpisodeEnding()) {
        System.out.println(String.format("Episode %d: %d rows cleared %d time", nbEpisode, problem.getRowsCleared(), step.time));
        step = problem.initialize();
        x_t = null;
        nbEpisode++;
      } else
        step = problem.step(action);
    }
  }
}